// 引入 SDK
const crawlab = require('crawlab-sdk')

const axios = require('axios')
const cheerio = require('cheerio')
const schoolName = '清华大学'
// 这是一个结果，需要为 object 类型
// const result = {name: 'crawlab'};

;(async () => {
  crawPage(`https://news.tsinghua.edu.cn/zhxw.htm`)
  for (let i = 438; i > 1; i--) {
    console.log(i)
    await new Promise((resolve) => {
      setTimeout(resolve, 500)
    })
    try {
      crawPage(`https://news.tsinghua.edu.cn/zhxw/${i}.htm`)
    } catch (e) {
      console.log(e)
    }
  }
  // 调用保存结果方法
  // await crawlab.saveItem(result);

  // // 关闭数据库连接，否则程序不会结束
    await new Promise((resolve) => {
      setTimeout(resolve, 5000)
    })
  console.log('down')
  await crawlab.close();
})()

async function crawPage(url) {
  const { data: html } = await axios(url)
  const $ = cheerio.load(html)
  $('.listl').each(async function (i, elem) {
    const that = $(elem)
    const time = that.children('.sj').text()
    let url = that.children('.jiequ').attr('href')
    const title = that.children('.jiequ').text()
    if (!url.match(/http(s?):\/\//)) {
      url = 'https://news.tsinghua.edu.cn/' + url
    }
    console.log(url)
    await crawlab.saveItem({
      time,
      url,
      title,
      schoolName
    })
  })
}

// https://news.tsinghua.edu.cn/info/1003/78759.htm
// https://news.tsinghua.edu.cn/zhxw/info/1003/78795.htm